// 对html数据进行处理，去标签

#include "errno_num.h"
#include "util.hpp"
#include "log.hpp"
#include <iostream>
#include <string>
#include <vector>
#include <boost/filesystem.hpp>

#include <mysql++/mysql++.h>




const int title_weight = 100;
const int content_weight = 1;

extern Log lg;
// const std::string title_st_label = "<title>";
// const std::string title_ed_label = "</title>";
// const char label_start = '<';
// const char label_end = '>';
// const std::string url_head = "https://www.boost.org/doc/libs/";

typedef struct doc_info
{
    std::string title;
    std::string content;
    std::string url;

} docInfo_t;

typedef struct inverted_elem
{
    uint64_t doc_id;
    std::string key_word;
    int weight = 0; // 权重
}invertedElem_t;

using invertedList_t = std::vector<invertedElem_t>;

bool enum_file(const std::string&, std::vector<std::string>*);
bool build_and_output_forward_index(const std::vector<std::string>&, std::vector<docInfo_t>*);
bool build_and_output_inverted_index(const std::vector<docInfo_t>&);
// bool output_file(const std::vector<docInfo_t>&, const std::string&);

int main()
{
    std::vector<std::string> files_path;
    // 获取所有html文件名
    if (!enum_file(files_folder, &files_path))
    {
        // std::cerr << "enum file error!" << std::endl;
        lg(Fatal, "enum file name error!");
        return Enum_err;
    }
    // 将每个文件中的内容处理成docInfo的格式
    std::vector<docInfo_t> files_info;
    if (!build_and_output_forward_index(files_path, &files_info))
    {
        // std::cerr << "parse file error!" << std::endl;
        lg(Fatal, "parse file error!");
        return Parse_err;
    }
    if (!build_and_output_inverted_index(files_info))
    {
        lg(Fatal, "parse inverted index error!");
        return 8;
    }
    // if (!output_file(files_info, output, con))
    // {
    //     // std::cerr << "output result error!" << std::endl;
    //     lg(Fatal, "output result error!");
    //     return Output_err;
    // }
    return 0;
}

bool enum_file(const std::string& files_folder, std::vector<std::string>* files_path)
{
    // 使用boost/filesystem中的path对象进行操作
    namespace bfs = boost::filesystem;
    bfs::path ps(files_folder);
    // 定义一个空的目录迭代器作为尾部标识
    bfs::recursive_directory_iterator end;
    for (bfs::recursive_directory_iterator it(ps); it != end; ++it)
    {
        if (!bfs::is_regular_file(*it)) // 判断是不是一个普通文件
            continue;
        if (it->path().extension() != ".html") // 如果是普通文件，判断是不是html文件
            continue;
        files_path->push_back(it->path().string()); // 是一个html文件，插入vector中
        // debug
        // std::cout << files_path->back() << std::endl;
    }
    return true;
}

static bool parse_title(const std::string& text, std::string* title)
{
    static const std::string title_st_label = "<title>";
    static const std::string title_ed_label = "</title>";
    size_t begin = text.find(title_st_label);
    if (begin == std::string::npos)
        return false;
    begin += title_st_label.size();
    size_t end = text.find(title_ed_label);
    if (end == std::string::npos)
        return false;
    if (begin > end)
        return false;
    *title = text.substr(begin, end - begin);
    return true;
}

static bool parse_content(const std::string& text, std::string* content)
{
    static const char label_start = '<';
    static const char label_end = '>';
    enum status
    {
        Label,
        Content
    };
    status s = Label;
    bool first_blank = true;
    for (auto c : text)
    {
        if (c == label_start)
        {
            s = Label;
            if (first_blank)
            {
                content->push_back(' ');
                first_blank = false;
            }
        }
        switch (s)
        {
        case Label:
            if (c == label_end)
                s = Content;
            break;
        case Content:
            first_blank = true;
            if (c == '\n') // ?
                c = ' ';
            content->push_back(c);
            break;
        default:
            return false;
        }
    }
    // debug
    // std::cout << (*content) << std::endl;
    return true;
}

static bool parse_url(const std::string& file, std::string* url)
{
    // 观察发现文件名和官网url的联系
    std::string url_head = "https://www.boost.org/doc/libs/" + boost_version;
    std::string url_tail = "/doc/html" + file.substr(files_folder.size()); // files_folder之后的路径就是官网中的html路径
    *url += url_head + url_tail;
    // std::cout << (*url) << std::endl;
    return true;
}


bool build_and_output_forward_index(const std::vector<std::string>& files_path, std::vector<docInfo_t>* files_info)
{
    mysqlpp::Connection con(false);
    con.connect(database.c_str(), host_name.c_str(), user_name.c_str(), passwd.c_str(), port);
    mysqlpp::Query(con.query("truncate table forward_index")).execute();

    // 所有的标签内容一定是被 "<>" 包裹起来的，可以利用这一特性用状态机进行数据侵清洗
    for (const auto& file : files_path)
    {
        std::string text;
        if (!ns_util::file_util::read_file(file, &text)) // 如果读取文件内容失败，输出信息，并跳过
        {
            // std::cerr << "read\"" << file << "\"error" << std::endl;
            lg(Warning, "read \"%s\" error", file.c_str());
            continue;
        }

        docInfo_t doc; // 解析文件内容
        if (!parse_title(text, &(doc.title)))
        {
            // std::cerr << "parse\"" << file << "\" title error" << std::endl;
            lg(Warning, "parse \"%s\" title error", file.c_str());
            continue;
        }
        if (!parse_content(text, &(doc.content)))
        {
            // std::cerr << "parse\"" << file << "\" content error" << std::endl;
            lg(Warning, "parse \"%s\" content error", file.c_str());
            continue;
        }
        if (!parse_url(file, &(doc.url)))
        {
            // std::cerr << "parse\"" << file << "\" url error" << std::endl;
            lg(Warning, "parse \"%s\" url error", file.c_str());
            continue;
        }
        // 分析完文件之后，将其插入数据库中，建立正排索引
        static std::string query_suf = "insert into forward_index (title, content, url) values (";
        mysqlpp::Query query = con.query();

        query << query_suf << mysqlpp::quote << doc.title << ", "
            << mysqlpp::quote << doc.content << ", "
            << mysqlpp::quote << doc.url << ")";
        mysqlpp::SimpleResult res = query.execute();
        if (!res)
        {
            std::cout << query.error() << std::endl;
            continue;
        }
        files_info->push_back(std::move(doc));
    }
    return true;
}
void build_inverted_helper(const std::string& text, std::unordered_map<std::string, int>& allWord_weight, bool is_title)
{
    std::vector<std::string> words;
    ns_util::jieba_util::get_instance()->cut_for_search(text, &words);
    std::unordered_map<std::string, int> word_map;
    for (auto& word : words)
    {
        // 首先进行忽略大小写操作：全部转成小写字母
        boost::algorithm::to_lower(word);
        word_map[word]++;
    }
    // 简单定义权重
    for (const auto& word_counter : word_map)
    {
        const auto& word = word_counter.first;
        const auto cnt = word_counter.second;
        if (is_title)
            allWord_weight[word] += title_weight * cnt;
        else
            allWord_weight[word] += content_weight * cnt;
    }
}

bool build_and_output_inverted_index(const std::vector<docInfo_t>& docs)
{
    std::unordered_map<std::string, invertedList_t> index;
    mysqlpp::Connection con(false);
    con.connect(database.c_str(), host_name.c_str(), user_name.c_str(), passwd.c_str(), port);
    mysqlpp::Query(con.query("delete from inverted_index")).execute();
    for (int i = 1; i <= docs.size(); ++i)
    {
        auto& doc = docs[i - 1];
        // 保存所有字符的权重信息
        std::unordered_map<std::string, int> allWord_weight;
        build_inverted_helper(doc.title, allWord_weight, true);
        build_inverted_helper(doc.content, allWord_weight, false);
        for (auto& word_info : allWord_weight)
        {
            const std::string& word = word_info.first;
            invertedElem_t elem;
            elem.doc_id = i;
            elem.key_word = word;
            elem.weight = word_info.second;
            index[word].push_back(std::move(elem));
        }
    }

    // 将每个invertedList_t转换成一个json
    for (auto& [str, list] : index)
    {
        Json::Value array;
        for (auto& e : list)
        {
            Json::Value v;
            v["doc_id"] = e.doc_id;
            v["weight"] = e.weight;
            array.append(std::move(v));
        }
        // 将转换的json串插入mysql中
        ns_util::json_util::change_out_sep(""); // 切换快速发送模式
        std::string json = ns_util::json_util::serialize(array);
        mysqlpp::Query q = con.query();
        q << "insert into inverted_index (word, file_list) values (";
        q << mysqlpp::quote << str << ", ";
        q << mysqlpp::quote << json << ");";
        mysqlpp::SimpleResult res = q.execute();
        if (!res)
            std::cout << "error: " << res.info() << std::endl;
    }
    return true;
}
